package com.odl.ds;

import com.odl.ds.pipeline.DownLoadPipeline;
import com.odl.ds.pipeline.UndrawPipeline;
import com.odl.ds.process.UndrawProcessor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;

import java.util.ArrayList;
import java.util.List;

@Slf4j
@SpringBootApplication
public class DsApplication implements CommandLineRunner {

    public static void main(String[] args) {
        SpringApplication.run(DsApplication.class, args);
    }

    @Override
    public void run(String... args) throws Exception {
        log.info("Spider Start...");
        List<Request> requests=new ArrayList<>();
        requests.add(new Request("https://undraw.co/illustrations"));
        for (int i=2;i<=39;i++){
            requests.add(new Request(String.format("https://undraw.co/illustrations/%s", i)));
        }

        Spider.create(new UndrawProcessor())
                .startRequest(requests)
                .addPipeline(new UndrawPipeline())
                .thread(5)
                .run();
    }
}